package com.aliasi.annotate.corpora;

import com.aliasi.chunk.Chunk;
import com.aliasi.chunk.Chunker;
import com.aliasi.chunk.Chunking;
import com.aliasi.io.FileExtensionFilter;
import com.aliasi.util.Files;
import com.aliasi.util.Streams;
import com.aliasi.util.Strings;
import com.aliasi.sentences.IndoEuropeanSentenceModel;
import com.aliasi.sentences.SentenceChunker;
import com.aliasi.sentences.SentenceModel;
import com.aliasi.tokenizer.IndoEuropeanTokenizerFactory;
import com.aliasi.xml.SAXWriter;
import java.io.*;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;

public class NewsToXml {

    public static void main(String[] args) throws Exception {
        SentenceModel sentenceModel = new IndoEuropeanSentenceModel(true,false);
        Chunker sentenceChunker = new SentenceChunker(IndoEuropeanTokenizerFactory.INSTANCE, sentenceModel);
        //File inDir = new File(args[0]);
        //File outDir = new File(args[1]);

        File inDir = new File("C:/Users/Diogo/Desktop/normalizehtml/txt");
        File outDir = new File("C:/Users/Diogo/Desktop/normalizehtml/xml");


        File[] inFiles = inDir.listFiles(new FileExtensionFilter(".txt"));
        for (File inFile : inFiles) {
            String name = inFile.getName();
            String baseName = name.substring(0,name.length()-".txt".length());
            File outFile = new File(outDir,baseName+".xml");
            convert(inFile,outFile,sentenceChunker);
        }
    }

    static void convert(File in, File out, Chunker sentenceChunker) throws Exception {
        byte[] bytes = Files.readBytesFromFile(in);
        CharsetDetector detector = new CharsetDetector();
        detector.setText(bytes);
        CharsetMatch match = detector.detect();
        String charset = match.getName();
        String text = match.getString();
        ArrayList<String> lines = new ArrayList<String>();
        Chunking chunking = sentenceChunker.chunk(text);
        Iterator<Chunk> it = chunking.chunkSet().iterator();
        for(int i = 0; it.hasNext(); i++){
            Chunk c = it.next();
            String sentence = "";
            if(i == 0){
                sentence = text.substring(c.start(), c.end());
                sentence = sentence.trim();
                String[] sentence_aux = sentence.split("\n");
                lines.add(sentence_aux[0]);
                StringBuilder sb = new StringBuilder();
                for(int j = sentence_aux.length - 1; j > 0; j--){
                    if(!sentence_aux[j].equals("")){
                        if(j == sentence_aux.length - 1){
                            sb.insert(0, sentence_aux[j].trim());
                        }else{
                            sb.insert(0, sentence_aux[j].trim() + " ");
                        }
                    } else{
                        break;
                    }
                }
                sentence = sb.toString();
            }else{
                //sentence = text.substring(c.start(),c.end()).trim().replace("\n", " ");
                sentence = text.substring(c.start(), c.end()).trim();
                sentence = sentence.replace("\n", " ");
            }
            lines.add(sentence);
        }
        System.out.println("file=" + in
                           + " charset=" + charset
                           + " title=" + lines.get(0));
        OutputStream outStream = new FileOutputStream(out);
        SAXWriter writer = new SAXWriter(outStream,Strings.UTF8);
        writer.startDocument();
        writer.characters("\n");
        writer.startSimpleElement("doc");
        writer.characters("\n");
        writer.startSimpleElement("chunk type=\"title\"");
        //writer.startSimpleElement("title");
        writer.characters(lines.get(0));
        writer.endSimpleElement("chunk");
        //writer.endSimpleElement("title");
        writer.characters("\n");
        writer.startSimpleElement("body");
        for (int i = 1; i < lines.size(); ++i) {
            if (lines.get(i).length() == 0) continue;
            writer.characters("\n");
            writer.startSimpleElement("chunk type=\"s\"");
            //writer.startSimpleElement("s");
            writer.characters(lines.get(i).trim());
            /*
            Chunking sentenceChunking = sentenceChunker.chunk(lines.get(i));
            
            for (Chunk chunk : sentenceChunking.chunkSet()) {
                writer.startSimpleElement("chunk","type","s");
                String sentenceText = lines.get(i).substring(chunk.start(),chunk.end());
                writer.characters(sentenceText);
                writer.endSimpleElement("chunk");
            }*/
            writer.endSimpleElement("chunk");
            //writer.endSimpleElement("s");
        }
        writer.characters("\n");
        writer.endSimpleElement("body");
        writer.characters("\n");
        writer.endSimpleElement("doc");
        writer.endDocument();
        Streams.closeOutputStream(outStream);
    }
}